#!/usr/bin/env python
# -*- coding: ascii -*-
# vim:ts=4:sw=4:softtabstop=4:smarttab:expandtab
#
# webtombogrep - a grep-like web based search tool
# Copyright (C) 2007  Chris Clark
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""
Web Tombo Grep
Recursively Search Tombo plain text and encrypted notes via web browser interface.
Can either use -d/--dir_name or just list the directory name without an flag

Usage: %prog [options] args
    -d, --dir_name=PATH: PATH Name Directory of notes to search, if omitted, args is used instead
    -i, --indexdir=PATH: PATH Name Directory containing search index
    -n, --note_encoding=ENCODING_NAME:    Encoding that notes are in
    -w, --no_webbrowser:    Do NOT open web browser
"""
##     -n, --note_encoding=ENCODING_NAME DEFAULT=cp1252:    Encoding that notes are in

# TODO on search result, either add search field/form (ideal) or edit javascript so it doesn't try and find it.

import os
import sys
import tempfile
import locale
import time
import re
import cgi
import urllib
import webbrowser
import textwrap
import difflib

try:
    # in python 2.6
    import json
except ImportError:
    try:
        # from http://code.google.com/p/simplejson
        import simplejson as json
    except ImportError:
        json=None
if json is None:
    raise ImportError()
    def dump_json(x):
        """dumb not safe!"""
        return repr(x)
else:
    dump_json = json.dumps


import optionparse

try:
    #raise ImportError
    # from https://github.com/trentm/python-markdown2 - much faster and more compliant than other implementation
    import markdown2
    have_markdown = True
except ImportError:
    markdown2 = None
    try:
        #raise ImportError
        # from http://sourceforge.net/projects/python-markdown
        import markdown
        have_markdown = True
    except ImportError:
        have_markdown = False


try:
    import pottymouth  # PottyMouth http://pypi.python.org/pypi/PottyMouth
except ImportError:
    pottymouth = None

try:
    import dmp_line_diffs
except ImportError:
    dmp_line_diffs = None


#import cherrypy # http://www.cherrypy.org/
## rhubarb tart # http://rhubarbtart.org/
## PySite (not Pysite, note case) 
##      http://www.programmingforums.org/forum/f43-python/t9028-pysite-web-development-framework.html
##      http://www.programmingforums.org/thread9028.html
try:
    #raise(ImportError)  ## debug
    import cherrypy
    from cherrypy.lib.static import serve_file
except ImportError, info:
    import dietcherrypy as cherrypy
    serve_file=cherrypy.serve_file

from pytombo.tombonote import caching_console_password_prompt, gen_static_password, remove_leading_path, file_contents, save_file_contents
from pytombo.chi_io import BadPassword
from pytombo.config import get_config
import pytombo.search
try:
    from pytombo.pyfsnote import PyFsFileNotes
except ImportError:
    PyFsFileNotes = None
try:
    from pytombo import ftsearch
except ImportError:
    ftsearch = None


def gen_href(url, visible_text=None):
    """Dumb take url and html-ify it"""
    if visible_text is None:
        visible_text = escape_html(url)
    return '<a href="%s">%s</a>' % (url, visible_text)

def removeWhiteSpace(in_str):
    """Stupid white space remover"""
    ## I think I have one of theses somewhere based on strip()/split()
    tmp_str = re.sub("\s+", " ", in_str)
    return tmp_str.strip()

def highlight_word(pat, in_str, emphasis_start=None, emphasis_end=None, highlight_whole_word=False, regex_search=False, ignore_case=False):
    """Simple routine to add emphasis to in_str to all occurrences of pat.
    highlight_word is not html aware so if pat and/or in_str are html, it is assumed that both pat, in_str are html "safe". I.e. caller should escape input strings
    If regex_search is True, pat is assumed to be a string in regex form
    
    Example prefix/postfix emphasis strings:
    
    # plain text
    prefix_str = '{'
    postfix_str = '}'
    prefix_str = '**'
    postfix_str = '**'
    
    # html
    prefix_str = '<bold>'
    postfix_str = '</bold>'
    prefix_str = '<b>'
    postfix_str = '</b>'
    prefix_str = '<emphasis>'
    postfix_str = '</emphasis>'
    prefix_str = '<span class="highlight">'
    postfix_str = '</span>'
    """
    if emphasis_start is None:
        #emphasis_start = "*" # "<emphasis>"
        emphasis_start = "<emphasis>"
    if emphasis_end is None:
        #emphasis_end = "*" # "</emphasis>"
        emphasis_end = "</emphasis>"
    ## TODO if highlight_whole_word: do something different - mostly useful as search is not wildcarded so only highlight real hits
    regex_options = re.MULTILINE
    if ignore_case:
        regex_options = regex_options | re.IGNORECASE
    if not regex_search:
        search_pattern = re.escape(pat)
    else:
        search_pattern = pat
    regex_obj = re.compile(search_pattern, regex_options)
    return regex_obj.sub(emphasis_start+"\g<0>"+emphasis_end, in_str)


AUTO_LINK_RE = re.compile(r"""
                        (                          # leading text
                          <\w+.*?>|                # leading HTML tag, or
                          [^=!:'"/]|               # leading punctuation, or 
                          ^                        # beginning of line
                        )
                        (
                          (?:https?://)|           # protocol spec, or
                          (?:www\.)                # www.*
                        ) 
                        (
                          [-\w]+                   # subdomain or domain
                          (?:\.[-\w]+)*            # remaining subdomains or domain
                          (?::\d+)?                # port
                          (?:/(?:(?:[~\w\+%-]|(?:[,.;:][^\s$]))+)?)* # path
                          (?:\?[\w\+%&=.;-]+)?     # query string
                          (?:\#[\w\-]*)?           # trailing anchor
                        )
                        ([\.,"'?!;:]|\s|<|$)       # trailing text
                           """, re.X)
## from http://pylonshq.com/WebHelpers/webhelpers/rails/text.py.html
# I have my own "hilight() routine as the Rails one does not handle regex or html tags
#def auto_link_urls(text, **href_options):
def auto_link_urls(text):
    # FIXME BUG only does single link per line - should be in markdown anyway....
    #extra_options = tag_options(**href_options)
    def handle_match(matchobj):
        all = matchobj.group()
        a, b, c, d = matchobj.group(1, 2, 3, 4)
        if re.match(r'<a\s', a, re.I):
            return all
        text = b + c
        if b == "www.":
            b = "http://www."
        #return '%s<a href="%s%s"%s>%s</a>%s' % (a, b, c, extra_options, text, d)
        return '%s<a href="%s%s">%s</a>%s' % (a, b, c, text, d)
    return re.sub(AUTO_LINK_RE, handle_match, text)

def auto_escape_slashes(text):
    """Automatically turn '\' into '\\' (or possible quote with backtick ` for markdown"""
    ###
    """Did briefly look at doing his with regex:
    
        match all slashes:
            \\
            
        match all strings containing slash(es) NOTE too greedy:
            \S*\\\S*
            
        match all strings containing slash(es) (not in indentd text) NOTE too greedy:
            ^(?!\s).*\S*\\\S*


        Only  matches ` quotes:

            `.+\\.+`
            
            
        does not handle ` quotes
            ^(?!\s).+\\
            ^(?!\s).*\\
    being lazy and slow do this my hand...
    """
    result = []
    for line in text.split('\n'):
        if line.startswith('\t') or line.startswith('    '):
            result.append(line)
        else:
            result.append(line.replace('\\', '\\\\')) ## WARNING screws up backticked text, e.g.: `c:\some'path`
        result.append('\n')
    return ''.join(result)
    
def auto_newline(text, line_length_cut_point=60):
    """Automatically turn short lines into <br>
    This is actually very similar to the built into PottyMouth 
    short_line_length wrapping code (developed independently!)"""
    result = []
    for line in text.replace('\r', '').split('\n'):
        if line.startswith('\t') or line.startswith('    '):
            result.append(line)
        else:
            result.append(line)
            if 1 < len(line) <= line_length_cut_point:
                result.append('  ')
        result.append('\n')
    return ''.join(result)
    
pretty_html = """
<STYLE TYPE="text/css">

<!--
body {
    font-family: verdana;
}
pre {
    background-color: lightgrey;
    border: 1px solid #5096B9;
    padding: 10 10 10 10;
}
h1 {
    background-color: #5096B9;
    color: white;
    padding: 10 10 10 10;
    text-align: center;
}
h2 {
    color: #5096B9;
}
h3 {
    color: #5096B9;
}
h4 {
    color: #5096B9;
}
h5 {
    color: #5096B9;
}
h6 {
    color: #5096B9;
}
span.error {
    color: #ff0000;
}
label.required {
    color: #0000ff;
}
-->

</STYLE>

""" # NOTE missing html end tag and body

# note PM does NOT (officially) support nested lists
# nor does it support code/pre blocks
# Odd behaviors with indented numbered lists with wrapped lines fixed in 2.1.0
# Bug with "smart" quotes, which corrupts SQL examples.
# Has issues with newlines between list items.
# But apart from code blocks and list nesting it appears to be more
# suitable as-is (no need for filters or extras)
def mypottymouth(instr):
    pottymouth.short_line_length = 60  # I've found 60 works better for my source text
    pottymouth.encoding = 'us-ascii'
    parser = pottymouth.PottyMouth(
      bold=False,  # avoid problem with "*" (/* ... */)
      italic=False,  # avoid problem with "_" (II_SYSTEM, II_TIMEZONE_NAME)
      definition_list=False,  # avoid problems with ":"
      emdash=False,
      ellipsis=False,
      smart_quotes=False,  # NOTE this flag has no impact!
    )
    result = parser.parse(instr)
    #result = '\n'.join([unicode(x, 'utf8') for x in result])
    #result = ''.join([str(x).decode('utf8').encode('us-ascii', 'xmlcharrefreplace') for x in result])
    #result = ''.join([str(x).decode(pottymouth.encoding).encode('us-ascii', 'xmlcharrefreplace') for x in result])
    result = ''.join([str(x) for x in result])
    return result

def mymarkdown(in_str, make_pretty=True, encoding=None):
    result=[]
    if make_pretty:
        result.append(pretty_html)
    ## TODO table of contents
    ## TODO footnotes
    ## auto hyperlink if missing <>
    ## Assume first line is title so "header" it
    temp_str = in_str
    #temp_str = auto_link_urls(in_str)  # this breaks links in indent (code/pre) blocks - maybe rely on pottymouth instead?
    temp_str = auto_escape_slashes(temp_str)
    temp_str = auto_newline(temp_str)
    if markdown2:
        if encoding and not isinstance(temp_str, unicode):
            # md2 does horrible things to non Unicode types and assumes utf8.
            try:
                temp_str = temp_str.decode('utf8')  # horrible hack, turns out I have a dirty mix of encodings in my notes collection
            except UnicodeDecodeError:
                temp_str = temp_str.decode(encoding)
        html_from_md = markdown2.markdown('#' + temp_str, extras=['code-friendly'])
    else:
        # fall back, NOTE some input text simply hangs this markdown library :-(
        html_from_md = markdown.markdown('#' + temp_str)
    if isinstance(html_from_md, unicode):
        html_from_md = html_from_md.encode('us-ascii', 'xmlcharrefreplace')
    result.append(html_from_md)
    return ''.join(result)

if dmp_line_diffs:
    def diff_files(file1content, file2content, fsfilename1='', fsfilename2='', difftype=None, ignore_case=False):
        difftype = difftype or 'html'
        
        if ignore_case:
            file1content = file1content.lower()
            file2content = file2content.lower()

        if difftype.startswith('html'):
            result = dmp_line_diffs.simple_html_diff(file1content, file2content, title=None, old_name=fsfilename1, new_name=fsfilename2)
        else:
            raise NotImplementedError('unknown diff type')

        return result
else:
    def diff_files(file1content, file2content, fsfilename1='', fsfilename2='', difftype=None, ignore_case=False):
        """quick and dirty differ, really would like to use the one from ReviewBoard
        
        can/will diff any file on the file system with NO checks
        
        uses batteries included Python difflib
        Also consider using http://code.google.com/p/google-diff-match-patch/wiki/API
        """
        
        mydiffcss = """
        <head>
            <meta http-equiv="Content-Type"
                  content="text/html; charset=ISO-8859-1" />
            <title></title>
            <style type="text/css">
                table.diff {font-family:Courier; border:medium;}
                .diff_header {background-color:#e0e0e0}
                td.diff_header {text-align:right}
                .diff_next {background-color:#c0c0c0}
                .diff_add {background-color:#aaffaa}
                .diff_chg {background-color:#ffff77}
                .diff_sub {background-color:#ffaaaa}
            </style>
        </head>
        """

        difftype = difftype or 'html'
        
        if ignore_case:
            file1content = file1content.lower()
            file2content = file2content.lower()
        
        file1content = file1content.splitlines()
        file2content = file2content.splitlines()
        
        """
        Other types:
            unified_diff
            ndiff
        """
        if difftype=='regular':
            d = difflib.Differ()
            diffs = list(d.compare(file1content, file2content))
            result = ''.join(diffs)
        elif difftype.startswith('html'):
            d = difflib.HtmlDiff()
            if difftype=='htmlfull':
                filename1, filename2 = 'f1', 'f2'
                #mytmpdiff = difflib.HtmlDiff().make_file(file1content, file2content, filename1, filename2)
                result = d.make_file(file1content, file2content, filename1, filename2)
            else:
                #import pdb ; pdb.set_trace()
                result = d.make_table(file1content, file2content)
                result = mydiffcss + result

        return result


current_dir = os.path.dirname(os.path.abspath(__file__))

##FIXME tombogrep.__doc__ use this to auto gen forms (in case help changes....)
class Root(object):
    def __init__(self, note_dir=None, note_encoding=None, fts_index_path=None):
        # Try and default from config file....
        self.config = get_config()
        language_name, self.system_encoding = locale.getdefaultlocale()  # FIXME used for file system encoding and it may well be different, see sys.getfilesystemencoding()
        if self.system_encoding is None:
            # uncommon, but TinyCore Linux ships with no locale by default
            self.system_encoding = sys.getdefaultencoding()
        self.notedir = note_dir
        if self.notedir is None or self.notedir == '':
            self.notedir = self.config['notedirectory']
        if '://' in self.notedir:
            self.note_store = PyFsFileNotes(self.notedir)
        else:
            self.note_store = pytombo.FileNotes(self.notedir)
        self.encoding = note_encoding
        if self.encoding is None or self.encoding == '':
            self.encoding = self.config['file_encoding']
            # should be not None now.....
            if self.encoding is None:
                default_encoding = sys.getdefaultencoding()
                if self.system_encoding != default_encoding:
                    ## we have to do our own (de-)encoding
                    self.encoding = self.system_encoding
        self.fts_index_path = fts_index_path ## todo pick up from config file
        if self.fts_index_path is None or self.fts_index_path == '':
            self.fts_index_path = self.config.get('fts_index_path', None)
        self.config['file_encoding'] = self.encoding
        global ftsearch
        if self.fts_index_path and ftsearch is None:
            # 'full text index provided but not full text system is installed'
            # for now simply raise normal error import error
            from pytombo import ftsearch
        pytombo.search.set_config(self.config)
        
        ## TODO mini-fi css/javascript and combine together (possibly include in html)
        ## TODO some of the js below is minified but not all of them
        self.htmltemplate = """
<html>
<head>

    <script type="text/javascript" src="/jquery/jquery.js"></script>
    <script type='text/javascript' src='/jquery/jquery.bgiframe.min.js'></script>
    <script type='text/javascript' src='/jquery/jquery.ajaxQueue.js'></script>
    <script type='text/javascript' src='/jquery/jquery.autocomplete.js'></script>
    <link rel="stylesheet" type="text/css" href="/jquery/jquery.autocomplete.css" />

    <link rel="search" type="application/opensearchdescription+xml" title="webtombo" href="%s">
    
    <script type="text/javascript">
        function form_setfocus() {document.myform.search_term.focus();}
        <!--
            function toggleLayer( whichLayer )
            {
              var elem, vis;
              if( document.getElementById ) // this is the way the standards work
                    elem = document.getElementById( whichLayer );
              else if( document.all ) // this is the way old msie versions work
                    elem = document.all[whichLayer];
              else if( document.layers ) // this is the way nn4 works
                    elem = document.layers[whichLayer];
              vis = elem.style;
              // if the style.display value is blank we try to figure it out here
              if(vis.display==''&&elem.offsetWidth!=undefined&&elem.offsetHeight!=undefined)
                    vis.display = (elem.offsetWidth!=0&&elem.offsetHeight!=0)?'block':'none';
              vis.display = (vis.display==''||vis.display=='block')?'none':'block';
            }
        //-->
    </script>
<STYLE TYPE="text/css">

<!--
body {
    font-family: verdana;
}
pre {
    background-color: lightgrey;
    border: 1px solid #5096B9;
    padding: 10 10 10 10;
}
h1 {
    background-color: #5096B9;
    color: white;
    padding: 10 10 10 10;
    text-align: center;
}
h2 {
    color: #5096B9;
}
h3 {
    color: #5096B9;
}
h4 {
    color: #5096B9;
}
h5 {
    color: #5096B9;
}
h6 {
    color: #5096B9;
}
span.error {
    color: #ff0000;
}
label.required {
    color: #0000ff;
}
#grepoptions {
    margin: 0px 20px 0px 20px;
    display: block;
    background-color: #ECE9D8;
    border: 1px solid #333333;
}
-->

</STYLE>

<script type="text/javascript">
$(function() 
{
    function format_opensearch(instr)
    {
        // format the "data" attribute from a single parsed (by parse_opensearch) result
        if (instr.description)
            return instr.suggested_word + " - " + instr.description;
        else
            return instr.suggested_word;
    }
    function parse_opensearch(data)
    {
        // Parse OpenSearch json results as defined in http://www.opensearch.org/Specifications/OpenSearch/Extensions/Suggestions/1.0
        data = eval(data); // todo this isn't safe! Ideally have SafeJsonEval()
        var parsed = [];

        for (var i=0; i < data[1].length; i++)
        {
            var suggested_word = jQuery.trim(data[1][i]);
            var desc = null;
            if (data.length >=3)
            {
                // looks like we have the (optional) description
                desc = jQuery.trim(data[2][i]);
            }
            if (suggested_word)
            {
                tempdict={'suggested_word':suggested_word, 'description':desc};
                parsed[parsed.length] = {
                    data: tempdict,
                    value: suggested_word,
                    result: suggested_word // actual autocomplete text to use (if selected)
                };
            }
        }
        return parsed;
    }

    var pytombo_autocomplete_on = %s;
    if (pytombo_autocomplete_on)
    {
        toggleLayer('grepoptions');
    
        $("#search_term").autocomplete('jquery_suggest',
            {
                multiple: true,
                multipleSeparator: ' ',
                parse: parse_opensearch,
                formatItem: format_opensearch
            }
        );
    }
}
);
</script>

</head>


<body onload="form_setfocus()">

%s

</body>
</html>

        """
        
        self.diff_form = """
<p>%s</p>
<form action="" method="POST" id="myform">
<table cellpadding="0" cellspacing="0" border="0">
    <tr><td><label>note </label></td><td><input type="text" name="n1" value="%s" size="100"/></td><td></td></tr>
    <tr><td><label>password 1</label></td><td><input type="password" name="p1" value="" size="100"/></td><td></td></tr>
    
    <tr><td><label>note </label></td><td><input type="text" name="n2" value="%s" size="100"/></td><td></td></tr>
    <tr><td><label>password 2</label></td><td><input type="password" name="p2" value="" size="100"/></td><td></td></tr>
    
</table>
<input type="submit" value="Submit"/>
</form>
"""

        self.view_form = """
<p>%s</p>
<form action="" method="POST" id="myform">
<table cellpadding="0" cellspacing="0" border="0">
    <tr><td><label>note</label></td><td><input type="text" name="note" value="%s" size="100"/></td><td></td></tr>
    <tr><td><label>password</label></td><td><input type="password" name="password" value="" size="100"/></td><td></td></tr>
</table>
<input type="submit" value="Submit"/>
</form>
"""

        self.form = """
<h1>PyTombo Search</h1>
TODO autocomplete="off" in form tag?
<form action="" method="%s" id="myform" name="myform">
<table cellpadding="0" cellspacing="0" border="0">

<tr><td><label>search_term</label></td><td><input type="text" id="search_term" name="search_term" maxlength=2048 size=55 value=""/></td><td></td></tr>
<tr><td><label>full text search</label></td><td><input type="%s" onClick="toggleLayer('grepoptions')" name="fts_search" %s/></td><td></td></tr>
</table>
<div id="grepoptions">
Options below only work if full text search is disabled
<table cellpadding="0" cellspacing="0" border="0">
<tr><td><label>regex_search</label></td><td><input type="checkbox" name="regex_search"/></td><td></td></tr>

<tr><td><label>ignore_case</label></td><td><input type="checkbox" name="ignore_case" CHECKED/></td><td></td></tr>

<tr><td><label>findonly_filename (includes *.chi)</label></td><td><input type="checkbox" name="findonly_filename"/></td><td></td></tr>

<tr><td><label>line_numbers</label></td><td><input type="checkbox" name="line_numbers"/></td><td></td></tr>

<tr><td><label>search_encrypted</label></td><td>
<select name="search_encrypted">
    <option value="off">No</option>
    <option value="on">Yes</option>
    <option value="only">Only</option>
</select>
</td><td></td></tr>

<tr><td><label>password</label></td><td><input type="password" name="password" value=""/></td><td></td></tr>

</table>
</div>

<input type="submit" value="Search"/>

<h2>full text search help</h2>
from Whoosh qparser.py: This parser handles:
<pre>
<code>
    - 'and', 'or', 'not'
    - grouping with parentheses
    - quoted phrase searching
    - wildcards at the end of a search prefix, e.g. help*
    - ranges, e.g. a..b
    - field_name:search_term (valid fields; title, content, path)
</code>
</pre>
For more details see <a href="http://packages.python.org/Whoosh/querylang.html">Whoosh query language synax</a>.

<h2>regex help</h2>
<pre>
<code>
Search for cat or dog:  cat|dog
Search for temporary or corporate (regex optional):   ora
Search for "pora" but NOT with a trailing "T":  pora[^t]
Search for "leve" and "deve" :  [dl]eve
Search for "ere", e.g. "different" and "were" but not "here":  [^h]ere
</code>
</pre>
<!--
<a href="javascript:toggleLayer('grepoptions');" title="Show/Hide grep/regex information">
grep/regex info</a><br>
-->
</form>
        """
        form_submit_type='POST' #'POST' #'GET'
        #form_submit_type='GET'   # FIXME This is for DEBUG only!
        if self.fts_index_path:
            fts_default='CHECKED'
            fts_input_type = 'checkbox'
            self.pytombo_autocomplete_on = '1'
        else:
            print 'ftsearch is disabled'
            self.pytombo_autocomplete_on = '0'
            fts_input_type = 'hidden'
            fts_default=''
        self.form = self.form % (form_submit_type, fts_input_type, fts_default)
        
    def index(self, search_term=None, fts_search=False, dir_name=None, line_numbers=False, regex_search=False, ignore_case=False, findonly_filename=False, search_encrypted=False, password=None):
        """plain text code and pre output, clone this routine (i.e. do not loose this) and add table support, table and also google style
        """
        
        form = self.htmltemplate % (cherrypy.url('webtomboopensearch.xml'), self.pytombo_autocomplete_on, self.form)
        
        dir_name=None # disable directory name specification in form
        if dir_name is not None:
            self.notedir = dir_name
        if dir_name is None and self.notedir is not None:
            dir_name = self.notedir
        
        #if search_term is None or  dir_name is None or  line_numbers is None or  regex_search is None or  ignore_case is None or  search_encrypted is None or  password is None:
        if search_term is None or  dir_name is None or  line_numbers is None or  regex_search is None or  ignore_case is None or  search_encrypted is None:
            form += '<a href="/indexnotes">indexnotes</a>'  # TODO put in side div with (in)visible toggle on it
            return form
        
        def force_bool(in_bool):
            if in_bool == True or in_bool == False:
                return in_bool
            else:
                return in_bool.lower() == 'on'
        regex_search = force_bool(regex_search)
        ignore_case = force_bool(ignore_case)
        fts_search = force_bool(fts_search)
        if fts_search:
            searchmodule=ftsearch
        else:
            #searchmodule = pytombo.search
            searchmodule = self.note_store  # NOTE this is really an object not a module
        
        if search_encrypted:
            # determine type of encrypted search, all or ONLY encrypted
            if search_encrypted == 'only':
                pass
            elif search_encrypted == 'off':
                search_encrypted = False
            else:
                # e.g. check box, 'on'
                # <input type="checkbox" name="search_encrypted"/>
                search_encrypted = True
        if search_encrypted:
            ## aggressive, no caching (http 1.1) headers
            cherrypy.response.headers['Expires'] = 'Sun, 19 Nov 1978 05:00:00 GMT'
            cherrypy.response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0'
            cherrypy.response.headers['Pragma'] = 'no-cache'
            
        result = self.search(searchmodule, search_term, dir_name, line_numbers, regex_search, ignore_case, findonly_filename, search_encrypted, password)
        return result
    index.exposed=True

    def search_text(self, searchmodule, search_term=None, dir_name=None, line_numbers=False, regex_search=False, ignore_case=False, findonly_filename=False, search_encrypted=False, password=None):
        # TODO FIXME findonly_filename must be implemented in search_func
        """plain text code and pre output, clone this routine (i.e. do not loose this) and add table support, table and also google style
        """
        search_iter=searchmodule.search_iter
        SearchCancelled=searchmodule.SearchCancelled ## Simplify do not need this? only search iter is interesting and index
            
        form = self.htmltemplate % (cherrypy.url('webtomboopensearch.xml'), self.pytombo_autocomplete_on, self.form)
        
        password_func = gen_static_password(password).gen_func()

        result = []
        
        result.append('<pre><code>')
        result.append('search_term:' + search_term)
        result.append('search_directory:' + dir_name)
        result.append('\n')
        
        search_location = dir_name
        
        try:
            for hit in search_iter(search_term, search_location, search_term_is_a_regex=regex_search, ignore_case=ignore_case, search_encrypted=search_encrypted, get_password_callback=password_func, findonly_filename=findonly_filename, index_name=self.fts_index_path):
                filename, hit_detail = hit
                filename = remove_leading_path(dir_name, filename)
                for result_hit_line, result_hit_text in hit_detail:
                    if line_numbers:
                        result.append('%s:%d:%s' % (filename, result_hit_line, result_hit_text))
                    else:
                        result.append('%s:%s' % (filename, result_hit_text))
        except SearchCancelled, info:
            result.append('search canceled because of', info)
        
        result.append('\n')
        result.append('</code></pre>')
        return self.htmltemplate % (cherrypy.url('webtomboopensearch.xml'), self.pytombo_autocomplete_on, ''.join(result))
        
    def search_html(self, searchmodule, search_term=None, dir_name=None, line_numbers=False, regex_search=False, ignore_case=False, findonly_filename=False, search_encrypted=False, password=None):
        """HTML output, clone this routine (i.e. do not loose this) and add table support, table and also google style, ala.:
        
            query issued at <b>Mon Oct 22 11:56:33 2007</b><br>
            Query: <b>setenv</b><br>
            Query time: <b>0.046000</b> seconds<br>
            <b>19</b> hits!


            <p>
                <a href="/static/dbms/informix.txt">informix</a>
                <br>
                hit text fragment
                <b> ...</b>
                <br>
                <font color="#008000">dbms\informix.txt - 12k</font> 
                <a href="highlight_note?note=dbms\informix.txt&search_terms=setenv">show note with search terms</a> 
                <a href="markdown_note?note=dbms\informix.txt">markdown note</a>
                <br>
            </p>
            
            <pre>
            <code>
            Elapsed time (search and render): <b>1.672000</b> seconds<br>
            </code>
            </pre>


        """
        search_iter=searchmodule.search_iter
        SearchCancelled=searchmodule.SearchCancelled ## Simplify do not need this? only search iter is interesting and index
        """
        if searchmodule is pytombo.ftsearch:
            print 'fts search on2'
        if searchmodule == pytombo.ftsearch:
            print 'fts search on3'
        """
        if searchmodule.__name__ == 'pytombo.ftsearch':
            fts_search_on=True
            search_location = self.note_store
        else:
            fts_search_on=False
            search_location = dir_name


        form = self.htmltemplate % (cherrypy.url('webtomboopensearch.xml'), self.pytombo_autocomplete_on, self.form)
        
        password_func = gen_static_password(password).gen_func()

        html_search_term = cgi.escape(search_term)
        result = []
        result_header = []
        
        if isinstance(dir_name, unicode):
            html_dir_name = dir_name.encode('ascii', 'xmlcharrefreplace')
        else:
            ## either in 7-bit ASCII or local encoding
            html_dir_name = dir_name
        result.append('<br>')
        result.append('<pre><code>')
        result.append('search_directory: %s\n' % html_dir_name)
        result.append('</code></pre>')
        result.append('\n')

        emp_prefix = "<b>"
        emp_postfix = "</b>"

        file_count = 0
        line_count = 0
        start_time = time.time()
        try:
            for hit in search_iter(search_term, search_location, search_term_is_a_regex=regex_search, ignore_case=ignore_case, search_encrypted=search_encrypted, get_password_callback=password_func, findonly_filename=findonly_filename, index_name=self.fts_index_path):
                file_count += 1
                full_filename, hit_detail = hit
                filename = remove_leading_path(dir_name, full_filename)
                basename_filename = os.path.basename(filename)
                if not isinstance(filename, unicode):
                    # probably in local encoding. We should convert to Unicode, in case there is a non-ASCII character. 
                    # e.g. latin1 under Windows, utf8 for most modern Linux distros
                    filename = filename.decode(self.system_encoding)
                    basename_filename = basename_filename.decode(self.system_encoding)
                if isinstance(filename, unicode):
                    html_filename_uri = urllib.quote(filename.encode('utf8'))
                    html_filename = filename.encode('ascii', 'xmlcharrefreplace')
                    html_basename_filename = basename_filename.encode('ascii', 'xmlcharrefreplace')
                else:
                    html_filename_uri = filename
                    html_filename = filename
                    html_basename_filename = basename_filename
                result.append('''
        <p>
            <a href="view?note=%s&html=true">%s</a>
            <br>
            ''' % (html_filename_uri, html_basename_filename))
                for result_hit_line, result_hit_text in hit_detail:
                    line_count += 1
                    if fts_search_on:
                        # Assume already highlighted and in html
                        processed_html_result_hit_text=result_hit_text
                    else:
                        ## TODO remote white space using removeWhiteSpace()? not needed for output prettyness but would be smaller (albeit with more processing)
                        processed_html_result_hit_text = cgi.escape(result_hit_text) ## allows html code to be highlighted, e.g. search term "<html>" can be used and highlighted
                        processed_html_result_hit_text = highlight_word(html_search_term, processed_html_result_hit_text, emphasis_start=emp_prefix, emphasis_end=emp_postfix, regex_search=regex_search, ignore_case=ignore_case) + emp_prefix + ' ...'+ emp_postfix
                    if line_numbers:
                        text_fragment = '%d:%s' % (result_hit_line, processed_html_result_hit_text)
                    else:
                        text_fragment = '%s' % (processed_html_result_hit_text, )
                    result.append('''
                %s
                <br>
                ''' % (text_fragment, ))
                ## <a href="file://%s" title="add localhost to trusted site for this to work!, for Firefox see http://kb.mozillazine.org/Firefox_:_Issues_:_Links_to_Local_Pages_Don't_Work">local file</a>
                ## <a href="file://localhost/%s" title="add localhost to trusted site for this to work!, for Firefox see http://kb.mozillazine.org/Firefox_:_Issues_:_Links_to_Local_Pages_Don't_Work">local file</a>
                #########
                ## genhref function instead of hard coded
                ## TODO size of file and viewing the file!
                ## TODO *.chi files
                ## TODO test search with bad password
                ## TODO line (table) wrap the fragment(s) (ala yahoo/google), join fragments into single file look up
                ## TODO shrink text size and increase text size for title and filesize
                try:
                    note_size = self.note_store.note_size(full_filename)
                except pytombo.PyTomboIO:
                    note_size = -1  # ignore the error

                result.append('''
            <font color="#008000">%s - %dk</font> 
            <a href="markdown?note=%s">markdown note</a>
            <a href="view?edit=true&note=%s">edit</a>
            <br>
        </p>    
            ''' % (html_filename, max(note_size / 1024, 1), html_filename_uri, html_filename_uri))
                
        except SearchCancelled, info:
            temp_error_str = emp_prefix + '<br>' + cgi.escape('search cancelled because of %s!' % info) + emp_postfix ## change colour to red?
            result.append(temp_error_str)
            result.insert(0, temp_error_str)
        end_time = time.time()
        search_time = end_time - start_time        
        
        result.append('\n')
        
        ##### Header start
        num_hits = file_count
        
        result_header.append('query issued at %s%s%s<br>\n' % (emp_prefix, time.asctime(time.localtime(time.time())), emp_postfix))
        result_header.append('Query: ')
        result_header.append(emp_prefix)
        result_header.append(html_search_term)
        result_header.append(emp_postfix)
        result_header.append('<br>\n')
        if search_time is not None:
            result_header.append('Query time: %s%.2f%s seconds<br>\n' % (emp_prefix, search_time, emp_postfix) )
        if num_hits == 0:
            result_header.append('Nothing found!')
            result_header.append('\n')
        else:
            #result_header.append('%s%d%s document hits!\n' % (emp_prefix, num_hits, emp_postfix))
            result_header.append('%s%d%s document hits! At least %s%d%s matches\n<br>' % (emp_prefix, num_hits, emp_postfix, emp_prefix, line_count, emp_postfix))
        
        result_header.extend(result)
        result = result_header
        ##### Header end

        #return self.htmltemplate % ''.join(result)
        #print '-'*54
        #print result
        """
        for x in result:
            if isinstance(x, unicode):
                print repr(x)
        print '-'*54
        print repr(result) ## search for cp1252 note that is in test corpus
        print 'DEBUG!', type(self.htmltemplate), type(result)
        """
        result = ''.join(result)
        #print 'DEBUG!', self.encoding, type(result) # , repr(result)
        if isinstance(result, unicode):
            result = result.encode('us-ascii', 'xmlcharrefreplace')
        elif self.encoding is not None and not isinstance(result, unicode):
            # probably in local encoding
            cherrypy.response.headers['Content-Type'] = 'text/html; charset=%s' % self.encoding
        #return self.htmltemplate % (cherrypy.url('webtomboopensearch.xml'), self.pytombo_autocomplete_on, result)
        return self.htmltemplate % (cherrypy.url('webtomboopensearch.xml'), self.pytombo_autocomplete_on, result)
        
        
    search = search_text
    search = search_html

    def list(self, s=None, recursive=True):
        """s is the subdir
        """
        ## aggressive, no caching (http 1.1) headers
        cherrypy.response.headers['Expires'] = 'Sun, 19 Nov 1978 05:00:00 GMT'
        cherrypy.response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0'
        cherrypy.response.headers['Pragma'] = 'no-cache'
        
        result = []
        if s:
            result.append('%s' % cgi.escape(s))
            result.append('</br>')
            result.append('</br>')
        
        if recursive:
            if recursive in ('n', 'f', 'no', 'false'):
                recursive = False
        if recursive:
            dirnames = []
            note_names = self.note_store.recurse_notes(sub_dir=s)
        else:
            dirnames, note_names = self.note_store.directory_contents(s)
        for dirname in dirnames:
            if s:
                note_sub_dir_urls_concat = s + '/' + dirname
            else:
                note_sub_dir_urls_concat = dirname
            tmp_link = '<a href="list?recursive=n&s=%s">%s</a></br>' % (note_sub_dir_urls_concat, dirname)  # TODO unicode pathnames
            result.append(tmp_link)
        for filename in note_names:
            if s:
                filename = s + '/' + filename  # TODO could end up outside (parent of) note dir (../../)
                disp_filename = remove_leading_path(s, filename)
            else:
                disp_filename = filename
            # TODO use a table? consider right-justifying links (or maybe move left?)
            tmp_html = """%s
            <a href="view?note=%s">Raw Text</a>
            
            <a href="view?note=%s&html=true">html</a>
            
            <a href="markdown?note=%s">markdown note</a>
            
            <a href="view?edit=true&note=%s">Edit</a>
            </br>
            """ % (cgi.escape(disp_filename), filename, filename, filename, filename, )
            result.append(tmp_html)
        cherrypy.response.headers['Content-Type'] = 'text/html'
        text_res = '\n'.join(result)  # TODO html escape......
        return text_res
    list.exposed = True
    
    def getnote(self, note=None, password=None):
        ## TODO make note path validation check/code a seperate routine so can be used for saving
        if note is None:
            return 'need a note passed!'
        tmp_note=note.replace('\\', '/')
        if '../' in tmp_note:
            return 'bad path for note'
        if self.notedir is None:
            return 'missing path for note'
        password_func = gen_static_password(password).gen_func()
        ## TODO central note io and unicode
        temp_str = self.note_store.note_contents(note, password=password)
        return temp_str
    
    def diff(self, n1=None, p1=None, n2=None, p2=None, ignore_case=None):
        """
        
                    pants/diff1.txt
                    pants/diff2.txt
                    
                    diff?n1=pants/diff1.txt&n2=pants/diff2.txt
                    http://localhost:8080/diff?n1=pants/diff1.txt&n2=pants/diff2.txt
        
        """
        n1 = n1 or ''
        n2 = n2 or ''
        if not n1 or not n2:
            return self.self.diff_form  % ('Missing note filename(s)', n1, n2)
        ## aggressive, no caching (http 1.1) headers
        cherrypy.response.headers['Expires'] = 'Sun, 19 Nov 1978 05:00:00 GMT'
        cherrypy.response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0'
        cherrypy.response.headers['Pragma'] = 'no-cache'
        
        cherrypy.response.headers['Content-Type'] = 'text/plain'
        try:
            temp_note1 = self.getnote(n1, p1)
            temp_note2 = self.getnote(n2, p2)
        except BadPassword, info:
            ## probably a bad or missing password
            #return a form
            cherrypy.response.headers['Content-Type'] = 'text/html'
            temp_str = 'ERROR: %s' % info
            temp_str = self.diff_form  % ('Missing note passwords', n1, n2)
            return temp_str
        
        #note_name_html = cgi.escape(note)
        #return temp_note1 + '-'*65+'\n'+  temp_note2
        # FIXME extract fname
        fsfilename1=''
        fsfilename2=''
        result = diff_files(temp_note1, temp_note2, fsfilename1=fsfilename1, fsfilename2=fsfilename2, difftype='htmlfull', ignore_case=ignore_case)
        cherrypy.response.headers['Content-Type'] = cherrypy.response.headers['Content-Type'].replace('text/plain', 'text/html')
        return result
        #if self.encoding is not None and not isinstance(temp_str, unicode):
    diff.exposed = True
    
    def view(self, note=None, password=None, perform_wrap=False, line_width=80, edit=False, html=False):
        if note is None:
            return self.view_form  % ('Missing note filename', '')
        ## aggressive, no caching (http 1.1) headers
        cherrypy.response.headers['Expires'] = 'Sun, 19 Nov 1978 05:00:00 GMT'
        cherrypy.response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0'
        cherrypy.response.headers['Pragma'] = 'no-cache'
        
        cherrypy.response.headers['Content-Type'] = 'text/plain'
        """
        return self.getnote(note, password)
        """
        try:
            temp_str = self.getnote(note, password)
        except BadPassword, info:
            ## probably a bad or missing password
            #return a form
            cherrypy.response.headers['Content-Type'] = 'text/html'
            temp_str = 'ERROR: %s' % info
            temp_str = self.view_form  % (temp_str, note)
            return temp_str
        except pytombo.PyTomboIO:
            expected_first_list = os.path.basename(note).split('.', 1)[0]
            temp_str = expected_first_list + '\n' + '** New file **'
        
        ## wrap does not work..
        if not edit and perform_wrap:
            temp_str2=[]
            for x in temp_str.split("\n"):
                temp_str2.append(textwrap.fill(x, line_width))
            #temp_str = textwrap.wrap(temp_str, line_width)
            temp_str = ''.join(temp_str2)
        if self.encoding is not None and not isinstance(temp_str, unicode):
            # probably in local encoding
            cherrypy.response.headers['Content-Type'] = 'text/plain; charset=%s' % self.encoding
        if edit:
            ## TODO use web browser info to create good size for text area, maybe unobstrosive javascript to make it scale dynamically as window is resized
            #<input type="hidden" name="note" value="%s"/>
            #<input type="text" name="note" value="%s" readonly/>
            edit_form="""
            <html>
            <head>
            <title>%s</title>
            </head>
            <body>
            <form accept-charset="utf-8" action="savenote" method="POST" id="myform" name="myform">
            
            <input type="text" name="note" value="%s" size=100 readonly/></br>
            <label>notetext</label>
            <br />
<!-- TODO There is way to get textarea to be 100 percent via CSS and/or javascript, however most browsers allow manual resizing of text area. See http://stackoverflow.com/questions/271067/how-can-i-make-a-textarea-100-width-without-overflowing-when-padding-is-present (using a textwrapper div) -->
<textarea rows="25" cols="80" name="notetext"  accept-charset="utf-8">
%s
</textarea>
            <br />
            <input type="submit" value="Save"/>
            </form>
            </body>
            </html>
            """
            cherrypy.response.headers['Content-Type'] = cherrypy.response.headers['Content-Type'].replace('text/plain', 'text/html')
            temp_str=edit_form % (cgi.escape(note), cgi.escape(note), cgi.escape(temp_str))
        elif html:
            html_view="""
            <html>
            <head>
            %s
            <title>%s</title>
            </head>
            <body>
            note: %s</br>
            
            <!-- Regular (GET) hrefs, consider icons/buttons or POST (via javascript?) -->
            <a href="view?note=%s">Raw Text</a>
            
            <a href="markdown?note=%s">markdown note</a>
            
            <a href="view?edit=true&note=%s">Edit</a>
            
            <hr/>
%s
            <hr/>
            
            <form action="" method="GET" id="myform" name="myform">
            <input type="hidden" name="note" value="%s"/>
            <input type="hidden" name="edit" value="True"/>
            <input type="submit" value="Edit"/>
            </form>

            </body>
            </html>
            """
            cherrypy.response.headers['Content-Type'] = cherrypy.response.headers['Content-Type'].replace('text/plain', 'text/html')
            note_name_html = cgi.escape(note)
            if pottymouth:
                if self.encoding is not None and not isinstance(temp_str, unicode):
                    try:
                        temp_str = temp_str.decode('utf8')  # horrible hack, turns out I have a dirty mix of encodings in my notes collection
                    except UnicodeDecodeError:
                        temp_str = temp_str.decode(self.encoding)
                html_note_contents = mypottymouth(temp_str)
            else:
                html_note_contents = '<pre><code>\n' + cgi.escape(temp_str) + '\n</code></pre>\n'
            
            # TODO replace native pathsep with '/'
            note_name_html_hack = note_name_html.replace('\\', '/')  # nasty Windows hack
            path_parts = note_name_html_hack.split('/')
            filename_only = path_parts[-1]
            note_sub_dir_urls = []
            note_sub_dir_urls_concat = ''
            for path_part in path_parts[:-1]:
                if note_sub_dir_urls_concat:
                    note_sub_dir_urls_concat += '/' + path_part
                else:
                    note_sub_dir_urls_concat = path_part
                tmp_link = '<a href="list?recursive=n&s=%s">%s</a>' % (note_sub_dir_urls_concat, path_part)  # TODO unicode pathnames
                note_sub_dir_urls.append(tmp_link)
                note_sub_dir_urls.append('/')
            note_sub_dir_urls.append(filename_only)
            note_sub_dir_urls = ' '.join(note_sub_dir_urls)
            temp_str = html_view % (pretty_html, note_name_html, note_sub_dir_urls, note_name_html, note_name_html, note_name_html, html_note_contents, note_name_html)
        return temp_str
    view.exposed = True
    
    def savenote(self, note=None, notetext=None, password=None):
        """two bugs:
            1) unicode chars looses existing note!
            2)
        """
        if note is None or notetext is None :
            return 'Missing note filename or text'
        
        tmp_note=note.replace('\\', '/')
        if '../' in tmp_note:
            return 'bad path for note'
        if self.notedir is None:
            return 'missing path for note'
        full_filename = os.path.join(self.notedir, note)
        ## TODO central note io and unicode
        
        print repr(cherrypy.request.headers)
        print cherrypy.request.headers
        for x in cherrypy.request.headers:
            print repr(x)
        print repr((note, full_filename, notetext))
        
        encoding = self.encoding
        if isinstance(notetext, unicode):
            try:
                notetext = notetext.encode(encoding)
            except UnicodeEncodeError:
                # horrible hack, turns out I have a dirty mix of encodings in my notes collection and sometimes have a real need for UTF8
                encoding = 'utf8'
                notetext = notetext.encode(encoding)

        # makedir path just in case...
        tmp_dirname = os.path.dirname(full_filename)
        pytombo.search.safe_mkdir(tmp_dirname)
        save_file_contents(notetext, filename=full_filename)
        result = 'note saving is experimental...it may not work!'
        if encoding != self.encoding:
            result = result + ' NOTE encoding is not expected due to Unicode characters, now using %s' % encoding
        return result
    savenote.exposed = True
    
    def markdown(self, note=None, password=None):
        if not have_markdown:
            return 'Markdown is not installed'
        
        ## aggressive, no caching (http 1.1) headers
        cherrypy.response.headers['Expires'] = 'Sun, 19 Nov 1978 05:00:00 GMT'
        cherrypy.response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0'
        cherrypy.response.headers['Pragma'] = 'no-cache'
        
        try:
            temp_str = self.getnote(note, password)
        except BadPassword, info:
            ## probably a bad or missing password
            #return a form
            cherrypy.response.headers['Content-Type'] = 'text/html'
            temp_str = 'ERROR: %s' % info
            temp_str = self.view_form  % (temp_str, note)
            return temp_str
        if self.encoding is not None and not isinstance(temp_str, unicode):
            # probably in local encoding
            cherrypy.response.headers['Content-Type'] = 'text/html; charset=%s' % self.encoding
        return mymarkdown(temp_str, encoding=self.encoding)
    markdown.exposed = True
    
    def indexnotes(self, forcecreate=False):
        """This method/web page is use to index all "*.txt" text notes
        (does not index encrypted .chi files).
        # TODO index in a seperate thread and allow server to serve files,
        can then add an indexnotes_progress url that can be used to check progress
        """
        filesystem_encoding = sys.getfilesystemencoding()  # file system encoding may be different to self.system_encoding (locale/system)
        # TODO check for fts index is none?
        def force_bool(in_bool):
            if in_bool == True or in_bool == False:
                return in_bool
            else:
                return in_bool.lower() == 'on'
        forcecreate = force_bool(forcecreate)
        index_name=self.fts_index_path
        index_does_not_exist=not os.path.exists(index_name)
        if index_does_not_exist or forcecreate:
            callbackfunc=None
            callbackfunc=ftsearch.example_progess_callback
            start_time = time.time()
            file_count = ftsearch.index_content(index_name, self.note_store, note_encoding=self.encoding, filesystem_encoding=filesystem_encoding, progess_callback=callbackfunc, clear_existing_index=forcecreate)
            end_time = time.time()
            search_time = end_time - start_time
            return 'indexed %d documents in %.2f seconds Using index %s <a href="/">start searching</a>' % (file_count, search_time, index_name)
            #import shutil
            #shutil.rmtree(index_name)
        else:
            return '''Index already exists, force re-index?
            <form action="" method="POST" id="myform">
    <label>forcecreate</label><input type="checkbox" name="forcecreate"/>
<input type="submit" value="Submit"/>
</form>
            '''
    indexnotes.exposed=True
    
    def webtomboopensearch_xml(self):
        """OpenSearch download URL for /webtomboopensearch.xml
        """
        cherrypy.response.headers['Content-Type']='application/opensearchdescription+xml'
        xml='''<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.1/"
                       xmlns:moz="http://www.mozilla.org/2006/browser/search/">
<ShortName>webtombo</ShortName>
<Description>webtombo Search</Description>
<InputEncoding>UTF-8</InputEncoding>
<Url type="application/opensearchdescription+xml"
     rel="self"
     template="%s" />
<Url type="application/x-suggestions+json" method="GET" template="%s">
  <Param name="search_term" value="{searchTerms}"/>
  <Param name="maxnum" value="9"/>
</Url>
<Url type="text/html" method="GET" template="%s">
  <Param name="search_term" value="{searchTerms}"/>
  <Param name="fts_search" value="on"/>
  <Param name="ignore_case" value="on"/>
</Url>
<moz:SearchForm>%s</moz:SearchForm>
</OpenSearchDescription>
''' % (cherrypy.url('webtomboopensearch.xml'), cherrypy.url('suggest'), cherrypy.url('/'), cherrypy.url('/'))
        return xml
    webtomboopensearch_xml.exposed=True
    
    def get_suggestions(self, search_term, maxnum=5):
        """Return suggestion. in OpenSearch like object.
        See 
            https://developer.mozilla.org/en/Creating_OpenSearch_plugins_for_Firefox
            https://developer.mozilla.org/en/Supporting_search_suggestions_in_search_plugins
        also see google.xml in firefox settings dir (and yacddb search)
        
        http://www.opensearch.org/Specifications/OpenSearch/Extensions/Suggestions/1.0
        http://www.opensearch.org/Specifications/OpenSearch/Extensions/Suggestions/1.1/Draft_1
        
        TODO BUG. start search for "rob" for some reason with my code "robots" does not show up,
        then adding "o" to get robo it starts to work..
        :-( occasionally it works...
        """
        
        maxnum=int(maxnum)
        search_term=search_term.strip() # remove white space
        orig_search_term_list=search_term.strip().split()
        search_term=orig_search_term_list.pop()  # remove last word/search term (can not do phrase lookups) and use that for lookup
        orig_search_term=''.join(orig_search_term_list)+' '
        
        result_list=[]
        result_list_desc=[]
        for tmpterm in ftsearch.gen_find_top_search_terms(self.fts_index_path, search_term, maxnum):
            result_list.append(orig_search_term+tmpterm[1]) ## could use ocunt for Descriptions fields, FireFox does not support this though (probably useful for search field with unobtrusive javascript helpers)
            result_list_desc.append('%d results'%tmpterm[0]) ## TODO make number have commas for seperators for easy reading
        result_dict=[search_term, result_list, result_list_desc]
        #result_dict=[search_term, result_list] ## DEBUG DO NOT USE, test missing (optional) description
        return result_dict
    
    def suggest(self, search_term, maxnum=5):
        """Return OpenSearch suggestion. in json
        See 
            https://developer.mozilla.org/en/Creating_OpenSearch_plugins_for_Firefox
            https://developer.mozilla.org/en/Supporting_search_suggestions_in_search_plugins
        also see google.xml in firefox settings/opensearch plugin dir
        
        http://www.opensearch.org/Specifications/OpenSearch/Extensions/Suggestions/1.0
        http://www.opensearch.org/Specifications/OpenSearch/Extensions/Suggestions/1.1/Draft_1
        
        
        ideal would be a javascript client that you can point at the opensearch xml file and it will work out the suggestion url (scheme)
        
        won't work could be hacked:
            http://woork.blogspot.com/2008/02/search-with-autosuggest-feature.html -- would need lots of hacking

        ojax http://ojax.sourceforge.net/demo.html appeasrs to handle json form that opensarch uses but is wierd (not documented, appeasr to need custom code)
        


        Scriptaculous Autocomplete with JSON and IE6 support
        The script.aculo.us Autocomplete control actually takes in an HTML partial 
        rather than a proper JSON or XML data format. 
        Furthermore, the keyboard navigation of the autocomplete widget fails in IE6.
        http://lebensold.net/creative-stuff/scriptaculous-autocomplete-with-json-and-ie6-support


        
        
        
        
        http://ex-libris.ca/?p=694
        http://search.cambridgelibraries.ca:2082/search
        http://www2.cambridgelibraries.ca/mike/yaz/opacsuggest.php?callback=jsonp1243802404633&_=1243802413645&q=can&limit=10&timestamp=1243802413644&searchtype=t
            jsonp1243802404633(["can a busy christian develop her spiritual life answers to questions women ask about spirituality", "can a giraffe cry or laugh", "can adults become human", "can amera parkway franklin boulevard to townline road pinebush road addendum to class environmental assessment", "can amera parkway franklin boulevard to townline road pinebush road class environmental assessment environmental study report", "can anybody hear me", "can asians think", "can diets be harmful", "can glance online", "can i change the way i look a teens guide to the health implications of cosmetic surgery makeovers and beyond"])
            
        appesrs to use jQuery plugin: Autocomplete
            
        wrong json format:
        jQuery plugin: Autocomplete
        http://bassistance.de/jquery-plugins/jquery-plugin-autocomplete/
        http://jquery.bassistance.de/autocomplete/demo/
        http://jquery.bassistance.de/autocomplete/demo/emails.php?q=pe
            [{ name: "Peter Pan", to: "peter@pan.de" }, ]
        
        json
        http://jquery.bassistance.de/autocomplete/demo/json.html
        http://jquery.bassistance.de/autocomplete/demo/emails.php?q=a&limit=10&timestamp=1243803530512
            [{ name: "Peter Pan", to: "peter@pan.de" }, { name: "Forneria Marconi", to: "live@japan.jp" }, { name: "Master Sync", to: "205bw@samsung.com" }, { name: "Donnie Darko", to: "dd@timeshift.info" }, { name: "Quake The Net", to: "webmaster@quakenet.org" }, ]
        
            
            
        http://blog.schuager.com/2008/09/jquery-autocomplete-json-apsnet-mvc.html
        http://www.it-eye.nl/weblog/2008/08/23/using-jquery-autocomplete-with-grails-and-json/
        http://farazt.blogspot.com/2008/10/autocomplete-using-jquery-mvc-and-json.html
        
        
        AutoComplete 1.2 Scriptaculous prototype with json
        http://www.cs.princeton.edu/~lbai/LF/ryan/doc/examples.html
            ["\r\nreabbreviate","reabbreviated","reabbreviates","reabbreviating","reach","reachability","reachable","reachably","reached","reacher","reaches","reaching","reacquired","react","reacted","reacting","reaction","reactionaries","reactionary","reactions","reactivate","reactivated","reactivates","reactivating","reactivation","reactive","reactively","reactivity","reactor","reactors","reacts","read","readability","readable","reader","readers","readied","readier","readies","readiest","readily","readiness","reading","readings","readjusted","readout","readouts","reads","ready","readying","Reagan","real","realest","realign","realigned","realigning","realigns","realism","realist","realistic","realistically","realists","realities","reality","realizable","realizably","realization","realizations","realize","realized","realizes","realizing","reallocate","really","realm","realms","realness","reals","realtor","ream","reanalyze","reanalyzes","reanalyzing","reap","reaped","reaper","reaping","reappear","reappeared","reappearing","reappears","reappraisal","reappraisals","reaps","rear","reared","rearing","rearrange","rearrangeable","rearranged","rearrangement","rearrangements","rearranges","rearranging","rearrest","rearrested","rears","reason","reasonable","reasonableness","reasonably","reasoned","reasoner","reasoning","reasonings","reasons","reassemble","reassembled","reassembles","reassembling","reassembly","reassessment","reassessments","reassign","reassigned","reassigning","reassignment","reassignments","reassigns","reassure","reassured","reassures","reassuring","reawaken","reawakened","reawakening","reawakens"]
        
        
        http://www.developer.com/design/article.php/10925_3719341_1
        http://www.developer.com/design/article.php/10925_3719341_4
        
        http://localhost:8080/suggest?search_term=fi&maxnum=9
        
        
        http://localhost:8080/?search_term=more&fts_search=on&ignore_case=on&password=
        http://localhost:8080/?fts_search=on&search_term=more
        
        
        http://localhost:8080/?fts_search=on&search_term={search_terms}
        
        
        """
        
        result_dict=self.get_suggestions(search_term, maxnum)
        result = dump_json(result_dict)
        ##TODO set content type (you would think json but google sends javascript) and encoding (utf8)
        return result
    suggest.exposed=True
    
    def jquery_suggest(self, q, limit=None, timestamp=None):
        """autocompete with api/params that 
            http://bassistance.de/jquery-plugins/jquery-plugin-autocomplete/ 
        expects but with opensearch json result set/type/format.
        """
        #import time
        #time.sleep(1)
        result_dict=self.get_suggestions(q, limit)
        result = dump_json(result_dict)
        ##TODO set content type (you would think json but google sends javascript) and encoding (utf8)
        return result
    jquery_suggest.exposed=True


class JQuery(object):
    @cherrypy.expose
    def jquery_autocomplete_js(self):
        local_filename='jquery.autocomplete.js'
        return serve_file(os.path.join(current_dir, 'jquerypytombo', local_filename),
                                  content_type='application/javascript')
    
    @cherrypy.expose
    def jquery_autocomplete_css(self):
        local_filename='jquery.autocomplete.css'
        return serve_file(os.path.join(current_dir, 'jquerypytombo', local_filename),
                                  content_type='text/css')
    
    @cherrypy.expose
    def indicator_gif(self):
        local_filename='indicator.gif'
        return serve_file(os.path.join(current_dir, 'jquerypytombo', local_filename),
                                  content_type='image/gif')
    
    @cherrypy.expose
    def jquery_ajaxQueue_js(self):
        local_filename='jquery.ajaxQueue.js'
        return serve_file(os.path.join(current_dir, 'jquerypytombo', local_filename),
                                  content_type='application/javascript')
    
    @cherrypy.expose
    def jquery_bgiframe_min_js(self):
        local_filename='jquery.bgiframe.min.js'
        return serve_file(os.path.join(current_dir, 'jquerypytombo', local_filename),
                                  content_type='application/javascript')
    
    @cherrypy.expose
    def jquery_js(self):
        local_filename='jquery.js'
        return serve_file(os.path.join(current_dir, 'jquerypytombo', local_filename),
                                  content_type='application/javascript')
    
        

    
def main(argv=None):
    if argv is None:
        argv = sys.argv
    
    # TODO FIXME if pythonw.exe under windows is the default launcher/runner for *.py files argv processing gets broken - add a check and maybe pop-up a win32 dialog saying so?
    opt, args = optionparse.parse(__doc__, argv[1:]) ## FIXME really need a NO GUI option here
    def usage_error(msg=''):
        raise SystemExit(optionparse.optionstring.replace("%prog",sys.argv[0]) + '\n' + msg) ## fixme make this stuff cass based in module, optionstring global is not cool...
    
    path_to_search = opt.dir_name
    if not path_to_search:
        try:
            path_to_search = args[0]
        except IndexError, info:
            path_to_search = None

    # index
    indexdir = opt.indexdir
    index_name = None
    if indexdir:
        index_name = os.path.join(indexdir, 'ftsindex')
        print 'index:', index_name
    
    print 'cherrypy.__version__', cherrypy.__version__
    
    
    server_port = 8080
    socket_host = '0.0.0.0'  ## fails with diet
    #socket_host = 'localhost'  ## only allow access from local machine
    #socket_host = ''  ## same as localhost (with dietcherry)
    
    
    """
    ## cherrypy v3 quickstart (no call backs allowed, need to thread locally)
    #cherrypy.quickstart(Root(form, self._webform_callback))
    
    ### cherrypy 3.0.2 does NOT have server.start_with_callback it is engine..?
    cherrypy.config.update({'server.socketPort':server_port})
    cherrypy.engine.start_with_callback(webbrowser.open, ('http://localhost:%d'%server_port,))
    """
    ### cherrypy 2.?.? (and dietcherypy)
    #cherrypy.config.update({'server.socketPort':server_port}) # maybe a 3.0 thing?
    cherrypy.config.update({'server.socket_port':server_port, 'server.socket_host': socket_host}) # CherryPy 3.1.2
    mywebapp=Root(note_dir=path_to_search, note_encoding=opt.note_encoding, fts_index_path=index_name)
    mywebapp.jquery=JQuery()
    if opt.no_webbrowser:
        cherrypy.quickstart(mywebapp)
    else:
        # style start for:
        #   CherryPy 3.1 (tested with 3.1.2)
        #   dietcherypy
        def launch_webbrowser():
            url='http://localhost:%d'%server_port
            webbrowser.open(url)
        cherrypy.engine.subscribe('start', launch_webbrowser) # CherryPy 3.1 api
        cherrypy.quickstart(mywebapp)


    return 0


if __name__ == "__main__":
    sys.exit(main())
